package com.wpw.dream.spider.processor;

import org.apache.log4j.Logger;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Request;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.pipeline.Pipeline;
import us.codecraft.webmagic.processor.PageProcessor;

public class StopableSpider extends Spider {

	private Logger logger = Logger.getLogger(StopableSpider.class);
	
	public StopableSpider(PageProcessor pageProcessor) {
		super(pageProcessor);
	}
	
	public static StopableSpider create(PageProcessor pageProcessor) {
        return new StopableSpider(pageProcessor);
    }
	

	@Override
	protected void processRequest(Request request) {
		logger.info("=============开始下载" + request.getUrl());
		Page page = downloader.download(request, this);
        if (page == null) {
            sleep(site.getSleepTime());
            onError(request);
            return;
        }
        // for cycle retry
        if (page.isNeedCycleRetry()) {
            extractAndAddRequests(page, true);
            sleep(site.getRetrySleepTime());
            return;
        }
        pageProcessor.process(page);
        if (page.getRequest().getExtra("stop") != null && (Boolean)page.getRequest().getExtra("stop")) {
        	
        	if (stat.get() != STAT_STOPPED) {
				this.stop();
				return;
    		}
        	
        }
        extractAndAddRequests(page, spawnUrl);
        if (!page.getResultItems().isSkip()) {
            for (Pipeline pipeline : pipelines) {
                pipeline.process(page.getResultItems(), this);
            }
        }
        //for proxy status management
        request.putExtra(Request.STATUS_CODE, page.getStatusCode());
        sleep(site.getSleepTime());
	}

	
}
